{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "from __future__ import division\n",
    "from __future__ import unicode_literals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expected cross entropy loss if the model:\n",
      "- learns neither dependency: 0.661563238158\n",
      "- learns first dependency:   0.519166699707\n",
      "- learns both dependencies:  0.454454367449\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "print(\"Expected cross entropy loss if the model:\")\n",
    "print(\"- learns neither dependency:\", -(0.625 * np.log(0.625) +\n",
    "                                      0.375 * np.log(0.375)))\n",
    "# Learns first dependency only ==> 0.51916669970720941\n",
    "print(\"- learns first dependency:  \",\n",
    "      -0.5 * (0.875 * np.log(0.875) + 0.125 * np.log(0.125))\n",
    "      -0.5 * (0.625 * np.log(0.625) + 0.375 * np.log(0.375)))\n",
    "print(\"- learns both dependencies: \", -0.50 * (0.75 * np.log(0.75) + 0.25 * np.log(0.25))\n",
    "      - 0.25 * (2 * 0.50 * np.log (0.50)) - 0.25 * (0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES']=''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Global config variables\n",
    "num_steps = 5 # number of truncated backprop steps ('n' in the discussion above)\n",
    "batch_size = 200\n",
    "num_classes = 2\n",
    "state_size = 4 #状态向量的维度\n",
    "learning_rate = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def gen_data(size=1000000):\n",
    "    X = np.array(np.random.choice(2, size=(size,)))\n",
    "    Y = []\n",
    "    for i in range(size):\n",
    "        threshold = 0.5\n",
    "        if X[i-3] == 1: #-3或-8为1的概率都为50%\n",
    "            threshold += 0.5\n",
    "        if X[i-8] == 1:\n",
    "            threshold -= 0.25\n",
    "        if np.random.rand() > threshold:\n",
    "            Y.append(0)\n",
    "        else:\n",
    "            Y.append(1)\n",
    "    return X, np.array(Y)\n",
    "\n",
    "# adapted from https://github.com/tensorflow/tensorflow/blob/master/tensorflow/models/rnn/ptb/reader.py\n",
    "# 生成一个batch的数据\n",
    "def gen_batch(raw_data, batch_size, num_steps):\n",
    "    raw_x, raw_y = raw_data\n",
    "    data_length = len(raw_x)\n",
    "\n",
    "    # partition raw data into batches and stack them vertically in a data matrix\n",
    "    batch_partition_length = data_length // batch_size  #将数据划分成batch_size个分区,求出每个分区的数据长度\n",
    "    data_x = np.zeros([batch_size, batch_partition_length], dtype=np.int32)\n",
    "    data_y = np.zeros([batch_size, batch_partition_length], dtype=np.int32)\n",
    "    for i in range(batch_size):\n",
    "        data_x[i] = raw_x[batch_partition_length * i:batch_partition_length * (i + 1)]\n",
    "        data_y[i] = raw_y[batch_partition_length * i:batch_partition_length * (i + 1)]\n",
    "    \n",
    "    # further divide batch partitions into num_steps for truncated backprop\n",
    "    epoch_size = batch_partition_length // num_steps\n",
    "\n",
    "    for i in range(epoch_size):\n",
    "        #在每个分区里面取一个分段(按num_steps计算)的数据,注意是每个分区\n",
    "        x = data_x[:, i * num_steps:(i + 1) * num_steps]\n",
    "        y = data_y[:, i * num_steps:(i + 1) * num_steps]\n",
    "        yield (x, y)\n",
    "\n",
    "def gen_epochs(n, num_steps):\n",
    "    for i in range(n):  #n表示有n个epoch,因为一次gen_batch会遍历完所有的数据\n",
    "        yield gen_batch(gen_data(), batch_size, num_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\"\"\"\n",
    "Placeholders\n",
    "\"\"\"\n",
    "\n",
    "x = tf.placeholder(tf.int32, [batch_size, num_steps], name='input_placeholder')\n",
    "y = tf.placeholder(tf.int32, [batch_size, num_steps], name='labels_placeholder')\n",
    "init_state = tf.zeros([batch_size, state_size])\n",
    "\n",
    "\"\"\"\n",
    "RNN Inputs\n",
    "\"\"\"\n",
    "\n",
    "# Turn our x placeholder into a list of one-hot tensors:\n",
    "# rnn_inputs is a list of num_steps tensors with shape [batch_size, num_classes]\n",
    "x_one_hot = tf.one_hot(x, num_classes) #编码后由二维矩阵变成三阶张量\n",
    "rnn_inputs = tf.unstack(x_one_hot, axis=1) #在维度1上做解绑,生成num_steps个batch_size x num_classes的矩阵"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " $$\\begin{align}\n",
    " &\\begin{bmatrix}1&2&3&4\\end{bmatrix}\\cdot\\begin{bmatrix}2&3\\\\4&5\\\\6&7\\\\8&9\\end{bmatrix} + \n",
    " \\begin{bmatrix}9&8&7\\end{bmatrix}\\cdot\\begin{bmatrix}6&5\\\\4&3\\\\2&1\\end{bmatrix}\\\\\n",
    " &=\\begin{bmatrix}1&2&3&4&9&8&7\\end{bmatrix}\\cdot\\begin{bmatrix}2&3\\\\4&5\\\\6&7\\\\8&9\\\\6&5\\\\4&3\\\\2&1\\end{bmatrix}\n",
    " \\end{align}$$\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Definition of rnn_cell\n",
    "\n",
    "This is very similar to the __call__ method on Tensorflow's BasicRNNCell. See:\n",
    "https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/rnn/python/ops/core_rnn_cell_impl.py#L95\n",
    "\"\"\"\n",
    "with tf.variable_scope('rnn_cell'):\n",
    "    W = tf.get_variable('W', [num_classes + state_size, state_size]) #[1,2]+[3,4] = [1,2,3,4]\n",
    "    b = tf.get_variable('b', [state_size], initializer=tf.constant_initializer(0.0))\n",
    "\n",
    "def rnn_cell(rnn_input, state):\n",
    "    with tf.variable_scope('rnn_cell', reuse=True): #reuse=True说明用的是同一个W,b\n",
    "        W = tf.get_variable('W', [num_classes + state_size, state_size])\n",
    "        b = tf.get_variable('b', [state_size], initializer=tf.constant_initializer(0.0))\n",
    "    #U x X和W x X要做一个拼接\n",
    "    #rnn_input大小:batch_size x num_classes\n",
    "    #state大小:batch_size x state_size\n",
    "    #W大小:(num_classes + state_size) x state_size\n",
    "    return tf.tanh(tf.matmul(tf.concat([rnn_input, state], 1), W) + b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Adding rnn_cells to graph\n",
    "\n",
    "This is a simplified version of the \"static_rnn\" function from Tensorflow's api. See:\n",
    "https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/rnn/python/ops/core_rnn.py#L41\n",
    "Note: In practice, using \"dynamic_rnn\" is a better choice that the \"static_rnn\":\n",
    "https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/ops/rnn.py#L390\n",
    "\"\"\"\n",
    "state = init_state\n",
    "rnn_outputs = []\n",
    "for rnn_input in rnn_inputs:\n",
    "    state = rnn_cell(rnn_input, state) #每一个rnn_input代表的是我们一个timestamp的输入\n",
    "    rnn_outputs.append(state) #我想这里主要是把num_steps个state求出来\n",
    "final_state = rnn_outputs[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "EPOCH 0\n",
      "Average loss at step 100 for last 100 steps: 0.642517189384\n",
      "Average loss at step 200 for last 100 steps: 0.626267742515\n",
      "Average loss at step 300 for last 100 steps: 0.621494550109\n",
      "Average loss at step 400 for last 100 steps: 0.6001739645\n",
      "Average loss at step 500 for last 100 steps: 0.563577776551\n",
      "Average loss at step 600 for last 100 steps: 0.531055501401\n",
      "Average loss at step 700 for last 100 steps: 0.526817025244\n",
      "Average loss at step 800 for last 100 steps: 0.524133172929\n",
      "Average loss at step 900 for last 100 steps: 0.522599186897\n",
      "\n",
      "EPOCH 1\n",
      "Average loss at step 100 for last 100 steps: 0.524967620075\n",
      "Average loss at step 200 for last 100 steps: 0.517552746832\n",
      "Average loss at step 300 for last 100 steps: 0.519161824286\n",
      "Average loss at step 400 for last 100 steps: 0.519567096829\n",
      "Average loss at step 500 for last 100 steps: 0.518155335188\n",
      "Average loss at step 600 for last 100 steps: 0.519228689671\n",
      "Average loss at step 700 for last 100 steps: 0.521272962689\n",
      "Average loss at step 800 for last 100 steps: 0.520025847554\n",
      "Average loss at step 900 for last 100 steps: 0.517731214464\n",
      "\n",
      "EPOCH 2\n",
      "Average loss at step 100 for last 100 steps: 0.524075783491\n",
      "Average loss at step 200 for last 100 steps: 0.520064674616\n",
      "Average loss at step 300 for last 100 steps: 0.519452514946\n",
      "Average loss at step 400 for last 100 steps: 0.517367443144\n",
      "Average loss at step 500 for last 100 steps: 0.519960871041\n",
      "Average loss at step 600 for last 100 steps: 0.519921580851\n",
      "Average loss at step 700 for last 100 steps: 0.522519366741\n",
      "Average loss at step 800 for last 100 steps: 0.519452979565\n",
      "Average loss at step 900 for last 100 steps: 0.519550951123\n",
      "\n",
      "EPOCH 3\n",
      "Average loss at step 100 for last 100 steps: 0.523972817361\n",
      "Average loss at step 200 for last 100 steps: 0.520993616283\n",
      "Average loss at step 300 for last 100 steps: 0.518277386427\n",
      "Average loss at step 400 for last 100 steps: 0.518152177036\n",
      "Average loss at step 500 for last 100 steps: 0.519574511647\n",
      "Average loss at step 600 for last 100 steps: 0.519184555709\n",
      "Average loss at step 700 for last 100 steps: 0.518691484928\n",
      "Average loss at step 800 for last 100 steps: 0.517750226855\n",
      "Average loss at step 900 for last 100 steps: 0.519773107767\n",
      "\n",
      "EPOCH 4\n",
      "Average loss at step 100 for last 100 steps: 0.527893611789\n",
      "Average loss at step 200 for last 100 steps: 0.519148774445\n",
      "Average loss at step 300 for last 100 steps: 0.519276213646\n",
      "Average loss at step 400 for last 100 steps: 0.520152277946\n",
      "Average loss at step 500 for last 100 steps: 0.517258580029\n",
      "Average loss at step 600 for last 100 steps: 0.518532965183\n",
      "Average loss at step 700 for last 100 steps: 0.520991978049\n",
      "Average loss at step 800 for last 100 steps: 0.517502208352\n",
      "Average loss at step 900 for last 100 steps: 0.520164375305\n",
      "\n",
      "EPOCH 5\n",
      "Average loss at step 100 for last 100 steps: 0.525851780176\n",
      "Average loss at step 200 for last 100 steps: 0.517591121197\n",
      "Average loss at step 300 for last 100 steps: 0.519051409066\n",
      "Average loss at step 400 for last 100 steps: 0.516216031015\n",
      "Average loss at step 500 for last 100 steps: 0.518410850763\n",
      "Average loss at step 600 for last 100 steps: 0.51945953846\n",
      "Average loss at step 700 for last 100 steps: 0.519771447182\n",
      "Average loss at step 800 for last 100 steps: 0.520950823724\n",
      "Average loss at step 900 for last 100 steps: 0.516678217053\n",
      "\n",
      "EPOCH 6\n",
      "Average loss at step 100 for last 100 steps: 0.523058773279\n",
      "Average loss at step 200 for last 100 steps: 0.521721841693\n",
      "Average loss at step 300 for last 100 steps: 0.520526286662\n",
      "Average loss at step 400 for last 100 steps: 0.518220171034\n",
      "Average loss at step 500 for last 100 steps: 0.521547440588\n",
      "Average loss at step 600 for last 100 steps: 0.51627296716\n",
      "Average loss at step 700 for last 100 steps: 0.518126632273\n",
      "Average loss at step 800 for last 100 steps: 0.517675009668\n",
      "Average loss at step 900 for last 100 steps: 0.517543879747\n",
      "\n",
      "EPOCH 7\n",
      "Average loss at step 100 for last 100 steps: 0.524554307759\n",
      "Average loss at step 200 for last 100 steps: 0.515849084556\n",
      "Average loss at step 300 for last 100 steps: 0.515828129947\n",
      "Average loss at step 400 for last 100 steps: 0.517297582328\n",
      "Average loss at step 500 for last 100 steps: 0.523019535244\n",
      "Average loss at step 600 for last 100 steps: 0.516739497483\n",
      "Average loss at step 700 for last 100 steps: 0.519933802187\n",
      "Average loss at step 800 for last 100 steps: 0.519996715188\n",
      "Average loss at step 900 for last 100 steps: 0.519734284282\n",
      "\n",
      "EPOCH 8\n",
      "Average loss at step 100 for last 100 steps: 0.523592251837\n",
      "Average loss at step 200 for last 100 steps: 0.518364751935\n",
      "Average loss at step 300 for last 100 steps: 0.519011733532\n",
      "Average loss at step 400 for last 100 steps: 0.519199455678\n",
      "Average loss at step 500 for last 100 steps: 0.520826595724\n",
      "Average loss at step 600 for last 100 steps: 0.517150171995\n",
      "Average loss at step 700 for last 100 steps: 0.51969401449\n",
      "Average loss at step 800 for last 100 steps: 0.51820019424\n",
      "Average loss at step 900 for last 100 steps: 0.519431493282\n",
      "\n",
      "EPOCH 9\n",
      "Average loss at step 100 for last 100 steps: 0.525503250659\n",
      "Average loss at step 200 for last 100 steps: 0.517508716285\n",
      "Average loss at step 300 for last 100 steps: 0.520220437348\n",
      "Average loss at step 400 for last 100 steps: 0.516647394001\n",
      "Average loss at step 500 for last 100 steps: 0.517130440474\n",
      "Average loss at step 600 for last 100 steps: 0.518613629937\n",
      "Average loss at step 700 for last 100 steps: 0.520247808695\n",
      "Average loss at step 800 for last 100 steps: 0.517506700456\n",
      "Average loss at step 900 for last 100 steps: 0.519363631606\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11c524850>]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD7CAYAAACG50QgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHqRJREFUeJzt3XmYFOW5NvD74QgqJiI7CgISwC0RxXy4RZ24gUbgkOCW\n5KBywvmMEkw8GvhMwmK4jsfjihfuR4maGEQ0LkEEBcaogIwIQYEB1FEWEUWEkR2m7++Pp1p6enqY\n7p6luqbu33XNBV1TVf1M1Vt3vfVW9YyRhIiIxEOTsAsQEZGGo9AXEYkRhb6ISIwo9EVEYkShLyIS\nIwp9EZEYOSDsAgDAzPTcqIhIHkhaLvMXTE+fZMF9jRkzJvQaVJNqimNdqim7r3wUTOiLiEj9U+iL\niMSIQn8/ioqKwi6hCtWUHdWUvUKsSzXVH8t3XKhOizBjIdQhIhIlZgZG9UauiIjUP4W+iEiMKPRF\nRGIkq9A3s35mVmpmK81sZDXzXGpmS83sPTP7c8r0CjN718wWmdnzdVW4iIjkrsYbuWbWBMBKAOcC\n+BRACYDLSZamzNMdwNMAfkiy3MzakNwYfK+c5KE1vIdu5IqI5Ki+buT2AbCK5Cck9wCYDGBg2jzD\nANxHshwAkoGfrCuXgkREpP5kE/odAaxJeb02mJaqJ4CjzexNM5trZn1TvnegmS0IpqefLEREpAFl\n8wvXMvXU08diDgDQHcBZADoDeMPMjg96/p1JfmZmRwGYbWZLSJbVqmoREclLNqG/Fh7kSZ3gY/vp\n88wjmQDwsZmtANADwEKSnwEAyTIzKwZwEoAqoT927Nhv/l9UVNRoPv0mIlJXiouLUVxcXKt1ZHMj\n918ArIDfyF0PYAGAK0guT5mnbzDtKjNrA2AhgBPhVwTbSe4Opr8FYGDqTeBged3IFRHJUT43cmvs\n6ZOsMLPhAGbC7wE8SnK5mY0DUELy7yRnmNkFZrYUwF4AN5L8ysxOA/CQmVUEy96aHvgiItJw9Lt3\nREQiSr97R0RE9kuhLyISIwp9EZEYUeiLiMSIQl9EJEYU+iIiMVIwob91a9gViIg0fgUT+qX6yJaI\nSL0rmNBftizsCkREGj+FvohIjCj0RURiRKEvIhIjBRP669YBO3aEXYWISONWMKHfvTuwYkXYVYiI\nNG4FE/rHHachHhGR+qbQFxGJEYW+iEiMKPRFRGKkYP5c4q5dRIsWwObNwIEHhl2RiEjhi/SfS2zW\nDOjaFVi1KuxKREQar4IJfUBDPCIi9U2hLyISIwp9EZEYUeiLiMRIwTy9QxI7dgCtWgHl5UDTpmFX\nJSJS2CL99A4AHHww0LEj8OGHYVciItI4FVToAxriERGpTwp9EZEYKbjQ/+53gfnzw65CRKRxKrjQ\nHzQIWLxYwS8iUh8KLvQPOQQYPx644QagAB4sEhFpVAou9AFgyBD/04lTp4ZdiYhI41JQz+mnmj0b\nGDbMb+rqt26KiFQV+ef0U51zjj/JM3Fi2JWIiDQeBdvTB4DSUuDMM/3f1q1DKExEpIA1qp4+ABxz\nDNC3LzBlStiViIg0DgUd+oA/t19WFnYVIiKNQ8GHfteuwMcfh12FiEjjoNAXEYmRSIS+hndEROpG\nwYd++/bA1q3+JSIitVPwoW/mvf1PPgm7EhGR6Cv40Ac0ri8iUlcU+iIiMaLQFxGJEYW+iEiMKPRF\nRGJEoS8iEiORCP127YBt2/SsvohIbUUi9M2ALl30rL6ISG1FIvQBDfGIiNQFhb6ISIwo9EVEYkSh\nLyISIwp9EZEYUeiLiMRIZEJfz+qLiNReZEJfz+qLiNReZEIf0BCPiEhtKfRFRGJEoS8iEiNZhb6Z\n9TOzUjNbaWYjq5nnUjNbambvmdmfU6ZfGSy3wsyG1KZYhb6ISO0cUNMMZtYEwEQA5wL4FECJmb1A\nsjRlnu4ARgI4jWS5mbUJprcEMBpAbwAGYGGw7JZ8ilXoi4jUTjY9/T4AVpH8hOQeAJMBDEybZxiA\n+0iWAwDJjcH0vgBmktxCcjOAmQD65VusQl9EpHayCf2OANakvF4bTEvVE8DRZvammc01s77VLLsu\nw7JZ07P6IiK1U+PwDnxYJh0zrKc7gLMAdAbwhpkdn+WyAICxY8d+8/+ioiIUFRVVLcSAww8H1q8H\nevTIonIRkUakuLgYxcXFtVpHNqG/Fh7kSZ3gY/vp88wjmQDwsZmtANAjmF6UtuycTG+SGvr707o1\n8OWXCn0RiZ/0DvG4ceNyXkc2wzslALqbWRczawbgcgAvps3zPIBzACC4idsDwEcAZgA438xaBDd1\nzw+m5a11a2DTptqsQUQkvmrs6ZOsMLPh8JuwTQA8SnK5mY0DUELy7yRnmNkFZrYUwF4AN5L8CgDM\n7I8A3oEP64wLbujmrVUr7+mLiEjujMw4xN6wRZgx2zpGjAC6dQN+/et6LkpEpMCZGUhmundarUh9\nIhfQ8I6ISG1ELvQ1vCMikr/IhX7y6R0REcldJENfwzsiIvmJXOhreEdEJH+RC3319EVE8hfJ0FdP\nX0QkP5EL/UMPBXbsAHbvDrsSEZHoiVzomwEtWwJffRV2JSIi0RO50Ac0xCMikq9Ihr6e4BERyU8k\nQ19P8IiI5Ceyoa+evohI7iIZ+hreERHJTyRDX8M7IiL5iWzoq6cvIpK7SIa+hndERPITydDX8I6I\nSH4iG/rq6YuI5C6Sod+qlXr6IiL5iGToq6cvIpKfSIb+wQcDJLB9e9iViIhESyRD30w3c0VE8hHJ\n0Ac0xCMiko/Ihr6e1RcRyV1kQ1/DOyIiuYt06KunLyKSm8iGvoZ3RERyF9nQ1/COiEjuIh366umL\niOQmsqGv4R0RkdxFNvQ1vCMikrtIh756+iIiuYls6Os3bYqI5M5Ihl0DzIy51rF7N3DIIf6vWT0V\nJiJSwMwMJHNKwMj29Js189+2WV4ediUiItER2dAHNMQjIpKrSIe+buaKiORGoS8iEiORDn0N74iI\n5CbSoa+evohIbiId+vpVDCIiuYl06OtXMYiI5Cbyoa+evohI9iId+hreERHJTaRDX8M7IiK5iXTo\nt2kDbNwYdhUiItER+dD/4ouwqxARiY5Ih36LFsDOncCuXWFXIiISDZEOfTP19kVEchHp0AeAtm0V\n+iIi2VLoi4jEiEJfRCRGFPoiIjGi0BcRiRGFvohIjCj0RURiRKEvIhIjCn0RkRhR6IuIxIiRDLsG\nmBnzrSORAJo1A3bsAJo2rePCREQKmJmBpOWyTFY9fTPrZ2alZrbSzEZm+P6VZva5mb0bfA1N+V5F\nMG2RmT2fS3HZaNJEf0xFRCRbB9Q0g5k1ATARwLkAPgVQYmYvkCxNm3UyyREZVrGNZO/al1q95BBP\nhw71+S4iItGXTU+/D4BVJD8huQfAZAADM8xX3SVGTpce+dC4vohIdrIJ/Y4A1qS8XhtMS/djM1ts\nZlPMrFPK9APNbIGZzTWzTCeLWlPoi4hkp8bhHWTuqaffdX0RwFMk95jZ/wXwOHw4CAA6k/zMzI4C\nMNvMlpAsS1/h2LFjv/l/UVERioqKsijNKfRFJA6Ki4tRXFxcq3XU+PSOmZ0KYCzJfsHrUQBI8rZq\n5m8CYBPJwzJ8bxKAl0g+lzY976d3AGDMGP933Li8VyEiEjn19fROCYDuZtbFzJoBuBzes09949Rb\nqAMBLAumHxYsAzNrA+D05Pfqknr6IiLZqXF4h2SFmQ0HMBN+kniU5HIzGweghOTfAYwwswEA9gDY\nBOCqYPFjATxkZhXBsrdmeOqn1hT6IiLZifyHswBg9mzglluAWg51iYhESr19OKvQqacvIpIdhb6I\nSIw0iuGdPXuAgw8Gdu/2X8sgIhIHsR3eadoUOPRQYNOmsCsRESlsjSL0AQ3xiIhkQ6EvIhIjCn0R\nkRhR6IuIxIhCX0QkRhT6IiIxotAXEYkRhb6ISIwo9EVEYkShLyISI43id+8AwK5dwLe/7f9avf8p\ndhGR8MX2d+8AwIEHAgcdBGzZEnYlIiKFq9GEPqAhHhGRmij0RURiRKEvIhIjCn0RkRhR6IuIxEij\nCv0uXYAFC8KuQkSkcDWq0L/qKmDJEmD69LArEREpTI0q9Js3Bx54ALj2WmDbtrCrEREpPI3mE7mp\nfvYz4IgjgNtvr7NViogUnFh/IjfV3XcDTzwBLF4cdiWSr61bgYcfDrsKkcanUYZ+u3bArbcCw4YB\nFRVhVyP5eP554JprgA0bwq5EpHFplKEPAFdfDbRoAYwYARTACJbkaOpU4NBDgWnTwq5EpHFptKFv\nBjz7LLBwoYI/ar7+GpgzBxg/HnjxxXBruftuoKQk3Bqk8KxbF91RhEYb+oD39GfM8Gf3r79+/8E/\nYwawfHnD1SbVmzYN+MEPgJ/+FJg9G9ixI5w61q8HRo0C7r8/nPeXwrRnD3DaacCkSWFXkp9GHfrA\nvuCfP997/Lt3V51nwgRgwADgllsavr5UJPDBB+HWAHjIhtmLmToVGDwYaNUK6N0bmDUrnDomTAAu\nugh46SU/0ONu6NDw9kUhee45YPNm4Jlnwq4kP40+9AHgsMOAmTOBDz8ETjgBePlln55IADfdBDz4\nIDB3rk8vLw+vzuJi4OijgbffDq+GRAI47zzgkkvCCf5t24BXXwUGDvTXAwaEM8SzZQvwyCM+vPOd\n7wCvv97wNRSSdeuAxx8Hnnwy/DrCfqrrnnv86m/+fGDjxnBryUcsQh/w4J82DbjzTuA3vwEuvBC4\n/HJg3jzgrbeAk08GioqAv/0tvBonTgT69vWb0Dt31u26s13fpEke9uXl4dwLefll4NRTvZcPePi/\n9JKfjBrSAw94L79rV+AnP/H7Q2FZuRI480zg6afDuwJ74gng/PP9GMq1BhL45S9rfxWbSABDhgDX\nXQe8/37t1pWv+fP9ibIrrgAuuMCfMosckqF/eRkNZ9cu8q67yF/9ity+fd/0KVPI885r0FK+sXo1\n2aoV+fXX5E9+Qo4cWXfrXrSIbNOGvOWW/c/35Zdku3bkwoXkli1kr17k+PF1V0c2Lr2UfPjhytOO\nO458++2Gq2H7drJDB3LJEn+9apW/3ru34WpIdd115ODB5Cmn+LaYPJmsqGi4908kyB49yHnzvE38\n4x+5Lf/66yTgP0dt3HMPedpp5H//tx8jYbjsMvLuu/3/U6aQF1wQTh1JQXbmlre5LlAfXw0d+tXZ\nvp1s2ZJct67u1rlpEzlnTs0H6e9+5ychkvzsMw/fugi6khJf1wMPkF27kk88Uf2811xT+cD89FNf\n5rHHal9HNrZtIw89lPz888rTR43y7dNQHniAvPjiytNOOIF8442GqyGpvNzb5Jo1Hr6vvOLhf/TR\n5IMPVu60ZPL66zWf7Gvy5pvkMcf4+48eTd54Y27LX3SRL9eypXcm8rFsmXdcVq3ydnL44d6ZaUif\nfFL5Z9i61dvrF180bB2pFPp1YOhQ8o476mZd77xDHnUU2b27HzSPPeZXGel27iTbtyeXL9837a9/\n9V7djh35v//cuWTbtuQLL/jrpUv9BDBrVtV5Fyzw3uxXX1WeXlrqtT37bP51ZOu558hzzqk6fe5c\n8nvfq5v3WL6cPPZY8vnnM39/zx6yWzcPulRjx5K//nXd1JBq1y4/Effp4yGe7r77yB//uPK0RMI7\nEv37+/4cPdpPDunKynzfHXaY7/t8DR1K3nab//+dd7zXn0hkt+w//+kBvWMHeckl5L335v7+u3eT\nJ59ceftMmOA/f7b27PErhTFjcn//pN/+tmobGDyYfOSR/NdZWwr9OjB7NnniibVbRyLhDbRNG78E\nTCTI117zoaNOncinn648/5//XHVYKZHwg71fP29US5fu/2phzRrvEY8c6b3im2/2wJ8+verP17Yt\n+f77+6bt3Ut+//vkn/6Ued0LF3q4JE8e9eWKK8j77686fe9ef/+PPqo8PZHwIZjRo307ffDB/tdf\nUuIntlGjfBt8+GHVee64g/zBD6pOf+89snPn7MOuOhUV5Pr1fhU3fjx5xBHkuef6Pm7d2vdj6s93\n3HGZT9JJpaW+3Xr1qrzstm3eju+6y4dDLr00v3q//tpPGp9+uq+mjh0rd1D252c/8/cnfVioZ8+q\n7XjFCm+zN95IjhjhV5w33eTDKJMnk8OH+9VC6rbfscOPpQULaq5h3jzfFuecQ37nO1WPv6QtW6rf\nv1u3+v5JbzNTppDnn1952oMPkoMG+YkmF6tX+0lk0CDvCGZDoV8HKiq8Mb333r5pGzd6GAwdSp5x\nhof58cd7zzS9kSxf7gfhd7/rB2S6efM8PMaP37fsqadm7nlu3uxDDT//uV8xtGrlPa709ywr8+8P\nH07eeiv5xz96j+attzL/jE8+SbZo4XW0bUs2b07+8If7P6ksWODzTpuW+fu7d5MPPeSB0L69B+fV\nV/sBP22aD5lVd0Dt3u0HfOfOVYd2koYO9e06fryf2K65xoc4unQh//M/vSferdu+cEo3a1blq557\n7yVPOqnyldTEib6+9JML6bX37OknjqTt2zPv43QffeShdvTR5IEHevs5+WTyF7/wnnDSmDF+wCfN\nmeNXJTWdaBIJ8n/+x9vtokX++qc/9XaTSHhwt2+/7x5FLiZNqjrU9ctf7gvy/Skr86DcvHlfnb16\n+RBV0uefe9sdMcLb9t13+374r//y4c7Bg8kf/Sjzfr3/fj/ZJ1VU+HH7yivkU0/5eq680q80/vIX\nf/+SEm8HZWWV1zV1KtmsGXn22d7JSVVa6lcpqfsmKX2I5+GHySOP9Pb/+9/XvI1Ib/933OHbavRo\n7+wNHOjT0+sYNKjyEJlCv4789rfeG6yo8B5Yu3bkVVf5GXzOHG+A06Z5A+7Th5wxwxvZ2Wf7wXXz\nzd7Tqs66dWTv3r7O5Ekgm5uEH3zg75e641et8uUnTsztZ1y92hv+Z5/50EA2NwbnzfMDZvJkcuVK\n3w7l5d7b6dnTe1Jvv02uXevb6aGHyBtu8KuYNm38q39/PwC3bvV1rllDnn46eeGFfnKtzuLFHgw3\n3+yBcO+9fiJKDcTx430YaNOmfdN27fJwaNuWLC7eNz2R8N7vf/yHv5440e9fZAr8pFGj/GvvXr8q\nOvJIP+BffTXz/C+/7L3A1q299nfe2X+72LHDt2OyAzB4cG77dcoU38ZXXeXtK3W8//bb87v5edZZ\nVYf2pk/3fVaT4cOrPpDwyCP7TiI7d3o43nxz7nUll+/c2a8K+vf38fYePfzm6uWXk9de620ledJJ\nuv12rz/ZE3/8cb8CLCnxNtuhAzlkiP+cgwZ52xk7tnK7SpUc4nnsMT/xrlrlx9URR1TfNpJmzfI2\ne/75fkyR3mb79/f17tnjbfX++70dTZxYuc0r9OvIkiW+w045xXvh776beb6KCg/7447zS/QpUzKP\n2Weydavv2ObNvXeerZ07vaeVDIdOnao+7VKf3nzTh4K6d/eD41vf8tczZ+5/uUTCT3ZPPukB36KF\nh26HDn5g1sXTKImEj7mecYb3vCZM8O1zwQWVe9RJ5eW+HS+7zAM/vfeXrqTE19erlz9F8tZbPmTR\ntq3fd0iqqPAg69rVT3C53JeZPdtPJitW5Hfjc+5c//k//rjy9G3bfFsvXrxv2gsv+M/TvTv5r//q\nVyOPP+7TZ83yjk3btlXb9M6dvv82bKi+js8/9/rTe+jbtvmJ6YMP/Epw0KDa7fuXXvKTy5QpPmyW\njYoKbxN/+INfSXfs6DeKk7Zs8f134okesvs7UZP+3kce6ZmReuX32mt+lZHpKmXZMj/5detGPvNM\n1au5nTvJvn29bV54oR9jmYbUFPp1aMgQ8n//t34fjdu71xtV+s3TbPzpTx641Y3DF7oNG/yAq+sn\nYioqyH/7N7JpUw+UmsZ8lywhzzyz5sAn/cC8+mrv+aYepNOn+9Xg4sV+IhkwwNdZ3VBVTZJDEtde\nm9/y1bnrLg/38nIfWjrqKL8iW7rUg2v0aB9Cu/hisqjIh6AmTMi8rsGDyUcfzfy9sjIfkkleRaW7\n6SYf6jrxRB96CsP69X4S7Nq15ntBNdm61Tt9qSeOpNGjfeh0yxa/Un7oIW+fbdqQd965/7H77dt9\nqOcPf6g61JOk0I+ZXG8UxcXevdn3+urKlCke1McfTw4blv0VXyZffOE3HGvzxE0m27d7b7RzZ79H\nku/jk6RfsQ0cWHnapk1+b6ZVK78/kRzCS1dW5ieU1avzf/+6sHRp9feA6srevT68edBBPuR29dX+\nFNGXX9bN+vMJ/Ub5l7NEwjB5sn+Sedgw/y2vtUHWfh2ZTJ8O7N0L9O9fu/Vs2uSfVu7Rw+ts0gQo\nKwMGDQLGjQMOP7xOym0UEgn/OuCAul93Pn85S6EvInlZvdp/90wy1Nq18xOBNByFvohIjOhv5IqI\nyH4p9EVEYkShLyISIwp9EZEYUeiLiMSIQl9EJEYU+iIiMaLQFxGJEYW+iEiMKPRFRGJEoS8iEiMK\nfRGRGFHoi4jEiEJfRCRGFPoiIjGSVeibWT8zKzWzlWY2MsP3rzSzz83s3eBraNr3VprZCjMbUpfF\ni4hIbmoMfTNrAmAigL4AjgdwhZkdk2HWySR7B1+PBcu2BDAawP8BcAqAMWbWos6qr2fFxcVhl1CF\nasqOaspeIdalmupPNj39PgBWkfyE5B4AkwEMzDBfpr/e0hfATJJbSG4GMBNAv7yrbWCFuJNVU3ZU\nU/YKsS7VVH+yCf2OANakvF4bTEv3YzNbbGZTzCz5/fRl11WzrIiINIBsQj9TDz79D9q+CKAryRMB\nzALwRA7LiohIA6nxD6Ob2akAxpLsF7weBYAkb6tm/iYAviTZ0swuB1BE8prgew8CmEPy6bRldCIQ\nEclDrn8YPZvQ/xcAKwCcC2A9gAUAriC5PGWeDiQ/C/4/CMBNJE8PbuS+A6A3/KriHQAnB+P7IiLS\nwA6oaQaSFWY2HH4TtgmAR0kuN7NxAEpI/h3ACDMbAGAPgE0ArgqW/crM/ggPewIYp8AXEQlPjT19\nERFpPEL/RG5NH/xqoBoeNbMNZrYkZVpLM5sZfKhsRkN/vsDMOpnZbDNbZmbvmdmIAqnrQDN728wW\nBXWNCaZ3NbP5QV1/NbMaryLruK4mwQcDXyyEeoIaPjazfwbbakEwLez918LMnjGz5Wa21MxOCbMm\nM+sZbJ93g3+3mNmIAthOvzGz981siZn9xcyaFUibuj447vLOhFBDP4cPftW3SUENqUYBeI3k0QBm\nA/h/DVzTXgA3kDwOwGkArgu2Tah1kdwF4IckTwJwIoALzewUALcBuDOoazOAf2/IugBcD2BZyuuw\n6wGABPxBhpNI9gmmhd2uJgB4meSxAHoBKA2zJpIrg+3TG8DJALYB+FuYNZnZEQB+BaA3yRPgw+BX\nIOQ2ZWbHB+/5ffixd7GZdUeu24pkaF8ATgUwPeX1KAAjQ6qlC4AlKa9LAbQP/t8BQGnI2+p5AOcV\nUl0AmsPv1/QB8DmAJin79ZUGrKMTgFcBFAF4MZj2RVj1pNRVBqB12rTQ9h+AbwP4MMP0gmhTAC4A\n8EbYNQE4AsAnAFrCA/9FAOeH2caD9xwM4OGU178HcBOA5blsq7CHd7L94FcY2pHcAAD0J5PahlWI\nmXWFn9nnw3duqHUFQymLAHwGD9sPAWwmmQhmWQs/cBrK3fDGz6C+1gC+CrGeJAKYYWYlZvaLYFqY\n+68bgI1mNikYTnnYzJqHXFOqywA8Ffw/tJpIfgrgTgCr4R8o3QLgXYTbxgHgfQBnBcM5zQFcBOBI\n5Litwg59fXirBmb2LQBTAVxPcisKYPuQTNCHdzrBe/nHZpqtIWoxsx8B2EByMfa1J0PVthXGdjud\n5PfhB+d1ZnZmSHUkHQB/fPo++nDKNvjVdehtysyaAhgA4JlgUmg1mdlh8F810wUe7IcAuDDDrA1a\nI8lS+BDTawBeBrAYPgyck7BDfy2AzimvOwH4NKRa0m0ws/aAfw4BfmnXoIIbRVMBPEnyhUKpK4lk\nOYDX4Ze6hwX3aICG3Y9nABhgZh8B+CuAcwDcA6BFSPV8I+h1geQX8OG5Pgh3/60FsIbkO8HrZ+En\ngUJoUxcCWEhyY/A6zJrOA/ARyU0kK+D3GE5HeG38GyQnkTyZZBGArwCsRI7bKuzQLwHQ3cy6mFkz\nAJfDx8/CkN47fBHB5w0AXAnghfQFGsBjAJaRnJAyLdS6zKxN8ukAMzsYfoAsAzAHwCUNXRfJm0l2\nJtkN3n5mk/x5WPUkmVnz4CoNZnYIfLz6PYS4/4IhgDVm1jOYdC6ApWHWlOIK+Ek7KcyaVgM41cwO\nMjPDvu0UapsCADNrG/zbGcAg+DbLbVs15I2Iam5O9IN/4ncVgFEh1fAU/Ky9C77Dr4bfxHktqO1V\nAIc1cE1nAKiAX8Itgo8p9gPQKuS6vhfUshjAEgC/C6YfBeBteM/jaQBNQ9iPZ2PfjdxQ6wneP7nv\n3ku27QLYf73gna3FAJ4D0KIAajoYfuP92ynTwq5pDPwG6RIAjwNoGnabCur6B3xsfxH8ybCct5U+\nnCUiEiNhD++IiEgDUuiLiMSIQl9EJEYU+iIiMaLQFxGJEYW+iEiMKPRFRGJEoS8iEiP/H24PpaZY\nksgoAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11be6e990>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\"\"\"\n",
    "Predictions, loss, training step\n",
    "\n",
    "Losses is similar to the \"sequence_loss\"\n",
    "function from Tensorflow's API, except that here we are using a list of 2D tensors, instead of a 3D tensor. See:\n",
    "https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/seq2seq/python/ops/loss.py#L30\n",
    "\"\"\"\n",
    "\n",
    "#logits and predictions\n",
    "with tf.variable_scope('softmax'):\n",
    "    #完成V x St部分\n",
    "    W = tf.get_variable('W', [state_size, num_classes])\n",
    "    b = tf.get_variable('b', [num_classes], initializer=tf.constant_initializer(0.0))\n",
    "    \n",
    "logits = [tf.matmul(rnn_output, W) + b for rnn_output in rnn_outputs]\n",
    "predictions = [tf.nn.softmax(logit) for logit in logits]\n",
    "\n",
    "# Turn our y placeholder into a list of labels\n",
    "y_as_list = tf.unstack(y, num=num_steps, axis=1) #干啥?打散成num_steps个y\n",
    "\n",
    "#losses and train_step\n",
    "losses = [tf.nn.sparse_softmax_cross_entropy_with_logits(labels=label, logits=logit) for \\\n",
    "          logit, label in zip(logits, y_as_list)]\n",
    "total_loss = tf.reduce_mean(losses)\n",
    "train_step = tf.train.AdamOptimizer(learning_rate).minimize(total_loss)\n",
    "\n",
    "\"\"\"\n",
    "Train the network\n",
    "\"\"\"\n",
    "def train_network(num_epochs, num_steps, state_size=4, verbose=True):\n",
    "    with tf.Session() as sess:\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "        training_losses = []\n",
    "        for idx, epoch in enumerate(gen_epochs(num_epochs, num_steps)): #训练多少个epoch\n",
    "            training_loss = 0\n",
    "            training_state = np.zeros((batch_size, state_size)) #初始人state当然是0咯\n",
    "            if verbose:\n",
    "                print(\"\\nEPOCH\", idx)\n",
    "                \n",
    "            for step, (X, Y) in enumerate(epoch):\n",
    "                tr_losses, training_loss_, training_state, _ = \\\n",
    "                    sess.run([losses,\n",
    "                              total_loss,\n",
    "                              final_state,\n",
    "                              train_step],\n",
    "                              feed_dict={x:X, y:Y, init_state:training_state})\n",
    "                training_loss += training_loss_\n",
    "                if step % 100 == 0 and step > 0: #此step为我们训练的step\n",
    "                    if verbose:\n",
    "                        print(\"Average loss at step\", step,\n",
    "                              \"for last 100 steps:\", training_loss/100)\n",
    "                    training_losses.append(training_loss/100)\n",
    "                    training_loss = 0\n",
    "\n",
    "    return training_losses\n",
    "training_losses = train_network(10,num_steps,state_size=state_size)\n",
    "plt.plot(training_losses)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
